package com.bj58.test

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.expr
import util.SparkReadUtil

/**
  * Created by 6v on 2018/11/5.
  */
object ZcmClientRead {

  val URL = "/home/hdp_lbg_ectech/resultdata/adsp/dp/ods/roi/click/"
  val SPLIT_WORD = '\u0001'
  val SPLIT_LINE = '\n'

  val COLUMN_SEQ = Seq("version","ua","remoteIp","forwardIp","lat","lon","coord","channel","city","refer","mac","cookie",
    "os","osversion","zp_uuid","netstatus","phone","uid","imei","idfa","time","net","logkey","ex1","ex2","ex3","ex4","ex5","ex6","ex7","ex8","content")

  val COLUMN_OUTPUT =Seq("version","ua","remoteIp","forwardIp","lat","lon","coord","channel","city","refer","mac",
    "os","osversion","zp_uuid","netstatus","phone","uid","imei","idfa","time","net","logkey","ex1","ex2","ex3","ex4","ex5","ex6","ex7","ex8","content")


  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .appName("BeeHiveRead")
      .master("local[1]")
      .getOrCreate()



    val path = "C:\\Users\\lenovo\\Desktop\\zcm_client_logger.log.2019-03-25-14.log"
    println(path)
    var data = SparkReadUtil.readFromFileZLog(spark,path,SPLIT_LINE,SPLIT_WORD,column_input =COLUMN_SEQ, COLUMN_OUTPUT)
//    data.map(row=>{
//
//    })
//    data = data.withColumn("uid",expr("split(uid,',')[1] "))
    data.filter(_.getAs[String]("os")=="14").show(20,false)
    println(data.count())
  /*  val rdd = spark.read.textFile(path).rdd.flatMap(_.split(SPLIT_LINE)).map(row => {
      println(row)
      val values = row.split(SPLIT_WORD).toSeq
//      println("====="+values.size)
      Row.fromSeq(values)
    })
    val schema = StructType(COLUMN_SEQ.map(fieldName => StructField(fieldName, StringType)))
    val data = spark.createDataFrame(rdd, schema)
   data.show(5)
    println("count:"+data.count())


    data.select(COLUMN_OUTPUT.head,COLUMN_OUTPUT.tail:_*).show(8)*/

  }


}
